{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-20T09:35:56.628130Z",
     "start_time": "2019-03-20T09:35:45.661384Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matchzoo version 1.0\n",
      "`ranking_task` initialized with metrics [normalized_discounted_cumulative_gain@3(0.0), normalized_discounted_cumulative_gain@5(0.0), mean_average_precision(0.0)]\n",
      "data loading ...\n",
      "data loaded as `train_pack_raw` `dev_pack_raw` `test_pack_raw`\n"
     ]
    }
   ],
   "source": [
    "%run init.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-20T09:35:56.633000Z",
     "start_time": "2019-03-20T09:35:56.630450Z"
    }
   },
   "outputs": [],
   "source": [
    "preprocessor = mz.models.ArcI.get_default_preprocessor(\n",
    "    filter_mode='df',\n",
    "    filter_low_freq=2,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-20T09:36:06.249211Z",
     "start_time": "2019-03-20T09:35:56.634788Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 5688.52it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 18841/18841 [00:04<00:00, 4144.24it/s]\n",
      "Processing text_right with append: 100%|██████████| 18841/18841 [00:00<00:00, 834035.69it/s]\n",
      "Building FrequencyFilter from a datapack.: 100%|██████████| 18841/18841 [00:00<00:00, 118659.36it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 133297.71it/s]\n",
      "Processing text_left with extend: 100%|██████████| 2118/2118 [00:00<00:00, 617770.23it/s]\n",
      "Processing text_right with extend: 100%|██████████| 18841/18841 [00:00<00:00, 672849.96it/s]\n",
      "Building Vocabulary from a datapack.: 100%|██████████| 404432/404432 [00:00<00:00, 2478649.32it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 8181.56it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 18841/18841 [00:04<00:00, 4103.55it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 114389.72it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 195405.74it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 112955.78it/s]\n",
      "Processing length_left with len: 100%|██████████| 2118/2118 [00:00<00:00, 603050.43it/s]\n",
      "Processing length_right with len: 100%|██████████| 18841/18841 [00:00<00:00, 754889.78it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 122/122 [00:00<00:00, 8076.03it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 1115/1115 [00:00<00:00, 4252.66it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 123603.15it/s]\n",
      "Processing text_left with transform: 100%|██████████| 122/122 [00:00<00:00, 116481.92it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 112869.84it/s]\n",
      "Processing length_left with len: 100%|██████████| 122/122 [00:00<00:00, 206332.70it/s]\n",
      "Processing length_right with len: 100%|██████████| 1115/1115 [00:00<00:00, 593332.78it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 237/237 [00:00<00:00, 8229.98it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2300/2300 [00:00<00:00, 4165.11it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 127761.65it/s]\n",
      "Processing text_left with transform: 100%|██████████| 237/237 [00:00<00:00, 158363.88it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 125427.75it/s]\n",
      "Processing length_left with len: 100%|██████████| 237/237 [00:00<00:00, 294184.68it/s]\n",
      "Processing length_right with len: 100%|██████████| 2300/2300 [00:00<00:00, 596444.86it/s]\n"
     ]
    }
   ],
   "source": [
    "train_pack_processed = preprocessor.fit_transform(train_pack_raw)\n",
    "dev_pack_processed = preprocessor.transform(dev_pack_raw)\n",
    "test_pack_processed = preprocessor.transform(test_pack_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-20T09:36:06.262937Z",
     "start_time": "2019-03-20T09:36:06.253350Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'filter_unit': <matchzoo.preprocessors.units.frequency_filter.FrequencyFilter at 0x7efff02aa0b8>,\n",
       " 'vocab_unit': <matchzoo.preprocessors.units.vocabulary.Vocabulary at 0x7efff069ed30>,\n",
       " 'vocab_size': 16675,\n",
       " 'embedding_input_dim': 16675}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_embedding = mz.datasets.embeddings.load_glove_embedding(dimension=300)\n",
    "term_index = preprocessor.context['vocab_unit'].state['term_index']\n",
    "embedding_matrix = glove_embedding.build_matrix(term_index)\n",
    "l2_norm = np.sqrt((embedding_matrix * embedding_matrix).sum(axis=1))\n",
    "embedding_matrix = embedding_matrix / l2_norm[:, np.newaxis]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainset = mz.dataloader.Dataset(\n",
    "    data_pack=train_pack_processed,\n",
    "    mode='pair',\n",
    "    num_dup=2,\n",
    "    num_neg=1\n",
    ")\n",
    "testset = mz.dataloader.Dataset(\n",
    "    data_pack=test_pack_processed\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "padding_callback = mz.models.ArcI.get_default_padding_callback(\n",
    "    fixed_length_left=10,\n",
    "    fixed_length_right=100,\n",
    "    pad_word_value=0,\n",
    "    pad_word_mode='pre'\n",
    ")\n",
    "\n",
    "trainloader = mz.dataloader.DataLoader(\n",
    "    dataset=trainset,\n",
    "    batch_size=20,\n",
    "    stage='train',\n",
    "    resample=True,\n",
    "    sort=False,\n",
    "    callback=padding_callback\n",
    ")\n",
    "testloader = mz.dataloader.DataLoader(\n",
    "    dataset=testset,\n",
    "    batch_size=20,\n",
    "    stage='dev',\n",
    "    callback=padding_callback\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-20T09:36:06.413530Z",
     "start_time": "2019-03-20T09:36:06.267256Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ArcI(\n",
      "  (embedding): Embedding(16675, 300)\n",
      "  (conv_left): Sequential(\n",
      "    (0): Sequential(\n",
      "      (0): ConstantPad1d(padding=(0, 2), value=0)\n",
      "      (1): Conv1d(300, 128, kernel_size=(3,), stride=(1,))\n",
      "      (2): ReLU()\n",
      "      (3): MaxPool1d(kernel_size=4, stride=4, padding=0, dilation=1, ceil_mode=False)\n",
      "    )\n",
      "  )\n",
      "  (conv_right): Sequential(\n",
      "    (0): Sequential(\n",
      "      (0): ConstantPad1d(padding=(0, 2), value=0)\n",
      "      (1): Conv1d(300, 128, kernel_size=(3,), stride=(1,))\n",
      "      (2): ReLU()\n",
      "      (3): MaxPool1d(kernel_size=4, stride=4, padding=0, dilation=1, ceil_mode=False)\n",
      "    )\n",
      "  )\n",
      "  (dropout): Dropout(p=0.9, inplace=False)\n",
      "  (mlp): Sequential(\n",
      "    (0): Sequential(\n",
      "      (0): Linear(in_features=3456, out_features=100, bias=True)\n",
      "      (1): ReLU()\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): Linear(in_features=100, out_features=1, bias=True)\n",
      "      (1): ReLU()\n",
      "    )\n",
      "  )\n",
      "  (out): Linear(in_features=1, out_features=1, bias=True)\n",
      ")\n",
      "Trainable params:  5578959\n"
     ]
    }
   ],
   "source": [
    "model = mz.models.ArcI()\n",
    "\n",
    "model.params['task'] = ranking_task\n",
    "model.params['embedding'] = embedding_matrix\n",
    "model.params['left_length'] = 10\n",
    "model.params['right_length'] = 100\n",
    "model.params['left_filters'] = [128]\n",
    "model.params['left_kernel_sizes'] = [3]\n",
    "model.params['left_pool_sizes'] = [4]\n",
    "model.params['right_filters'] = [128]\n",
    "model.params['right_kernel_sizes'] = [3]\n",
    "model.params['right_pool_sizes'] = [4]\n",
    "model.params['conv_activation_func'] = 'relu'\n",
    "model.params['mlp_num_layers'] = 1\n",
    "model.params['mlp_num_units'] = 100\n",
    "model.params['mlp_num_fan_out'] = 1\n",
    "model.params['mlp_activation_func'] = 'relu'\n",
    "model.params['dropout_rate'] = 0.9\n",
    "\n",
    "model.build()\n",
    "\n",
    "print(model)\n",
    "print('Trainable params: ', sum(p.numel() for p in model.parameters() if p.requires_grad))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-20T09:36:06.422264Z",
     "start_time": "2019-03-20T09:36:06.415605Z"
    }
   },
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adadelta(model.parameters())\n",
    "\n",
    "trainer = mz.trainers.Trainer(\n",
    "    model=model,\n",
    "    optimizer=optimizer,\n",
    "    trainloader=trainloader,\n",
    "    validloader=testloader,\n",
    "    validate_interval=None,\n",
    "    epochs=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-20T09:37:59.341616Z",
     "start_time": "2019-03-20T09:36:06.425086Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1383202d84034e2187df1c442d53e9ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-102 Loss-0.950]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.4339 - normalized_discounted_cumulative_gain@5(0.0): 0.5039 - mean_average_precision(0.0): 0.4654\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0dd9780c0daf43a8ba8e22d777a98c6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-204 Loss-0.846]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5491 - normalized_discounted_cumulative_gain@5(0.0): 0.6054 - mean_average_precision(0.0): 0.5702\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9e94f446dab4c4eab8fb091bdb886a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-306 Loss-0.722]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5355 - normalized_discounted_cumulative_gain@5(0.0): 0.5959 - mean_average_precision(0.0): 0.5558\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d39644bd700c40c49a16eab130ba33e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-408 Loss-0.668]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5323 - normalized_discounted_cumulative_gain@5(0.0): 0.597 - mean_average_precision(0.0): 0.5522\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fef2a18195274a2c81edc211cf03c89f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-510 Loss-0.590]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5602 - normalized_discounted_cumulative_gain@5(0.0): 0.6138 - mean_average_precision(0.0): 0.5738\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d47a7d9698b042a980887e97038093a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-612 Loss-0.524]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5609 - normalized_discounted_cumulative_gain@5(0.0): 0.6304 - mean_average_precision(0.0): 0.5848\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80b8fefe29d247a4b364dd05f49aa42d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-714 Loss-0.461]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5742 - normalized_discounted_cumulative_gain@5(0.0): 0.6483 - mean_average_precision(0.0): 0.6031\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f86e6549b0a946089164cced4f8e9c82",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-816 Loss-0.364]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5794 - normalized_discounted_cumulative_gain@5(0.0): 0.648 - mean_average_precision(0.0): 0.6049\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "09e42836162642edb0c27da2186fc022",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-918 Loss-0.298]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5267 - normalized_discounted_cumulative_gain@5(0.0): 0.6109 - mean_average_precision(0.0): 0.5608\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff654ea3430f4a21806f80fa9a2e9122",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-1020 Loss-0.280]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5961 - normalized_discounted_cumulative_gain@5(0.0): 0.6492 - mean_average_precision(0.0): 0.6111\n",
      "\n",
      "Cost time: 1278.5495643615723s\n"
     ]
    }
   ],
   "source": [
    "trainer.run()"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
